/*
 ==============================================================================
 Name        : WholeIO.c
 Version     : 4.1
 Description : Coding sample for utilizing the HCP HTTP Whole Object I/O feature.
               This feature allows both object data and the custom-metadata to
               be transferred with HCP with one HTTP interaction instead of two.

               The benefit of Whole Object I/O is to reduce the network band-
               width required to transfer objects to HCP by reducing the number
               of HTTP interactions required to write an object and its custom-
               metadata.

               This sample uses the libcurl open-source library to perform the
               HTTP communication to HCP.

               NOTE: This feature was introduced in HCP version 4.1.

 License     :
               Copyright (c) 2011 Hitachi Data Systems, Inc.

               Permission is hereby granted to  this software and associated
               documentation files (the "Software"), subject to the terms and
               conditions of the Sample Source Code License (SSCL) delivered
               with this Software. If you do not agree to the terms and
               conditions of the SSCL,

                 (i)  you must close this file and delete all copies of the
                      Software, and
                 (ii) any permission to use the Software is expressly denied.

 ==============================================================================
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <unistd.h>
#include <errno.h>
#include <fcntl.h>
#include <ctype.h>

#include <curl/curl.h>

#include <openssl/evp.h>

#include "hcpUtils.h"

#define INPUT_DATA_FILE_NAME "wholeio.txt"
#define OUTPUT_FILE_NAME "wholeio.txt.fromHCP"

#define INPUT_CM_FILE_NAME "wholeio.txt.cm"
#define OUTPUT_CM_FILE_NAME "wholeio.txt.fromHCP.cm"

#define HCP_PROTOCOL "https"
#define HCP_FILE_PATH "rest/examples"
#define HCP_WHOLE_IO_PARAM "type=whole-object"

/**
 * This sample has code to perform writing the object data first or the
 *   custom-metadata first.  To write the custom metadata first, un-
 *   comment the following line.
 */
//#define PERFORM_CM_FIRST

/*
 * This item structure is used to keep track of Whole I/O context when
 * reading and writing content with Whole I/O.  It is passed into various
 * CURL callback functions to update/use information stored in the structure.
 */
typedef struct {
	FILE *localDataFile;
	FILE *localCMFile;
	off_t dataFileSize;
	off_t cmFileSize;
	off_t contentLength;
	int bFirstFileComplete;
	int bCustomMetadataFirst;
} item_t;

/**
 * Function to process the HTTP headers returned from requests. This implementation
 * dumps out the header to standard out (great for debugging/development), and it
 * handles processing any headers that need attention.
 *
 * One header watched for in this implementation is the X-HCP-Size header that is
 * returned on a GET request.  This value is needed when processing the HTTP stream.
 *
 * If provided to the libcurl CURLOPT_HEADERFUNCTION, it will be executed.
 */
size_t myHeaderProcessor(void *ptr, size_t size, size_t nmemb, void *userdata) {
	char *buff = malloc((size*nmemb)+1);
	char *inPtr = (char*)ptr;

	int outIdx = 0;
	int inIdx = 0;

	// Strip out any control characters.
	for (; inIdx < size*nmemb; inIdx++) {
		// Strip out any special chars.  I'm a Linux/Unix biggot.
		if (!iscntrl(inPtr[inIdx]))
			buff[outIdx++] = inPtr[inIdx];
	}

	buff[outIdx] = '\0';  // Null terminate.

	// See if we need to fill in information about the data.
	// If we were passed in userdata, need to look for some header
	// values and fill in the item structure.
	if (NULL != userdata) {
		item_t *myItem = (item_t *)userdata;

		if (buff == strstr(buff, "X-HCP-Size:")) {

			/*
			 * Encountered the X-HCP-Size header on the response.
			 */
			int tmpsize;
			sscanf(buff, "X-HCP-Size: %d", &tmpsize);

			// Save it off in the item structure. Needed for later processing.
			myItem->dataFileSize = (off_t)tmpsize;

			// Compute the custom metadata size if possible. Can't compute it if
			//   "X-HCP-Size" comes before "Content-Length" in the HTTP response
			//   header.
			if (0 != myItem->contentLength)
				myItem->cmFileSize = myItem->contentLength - myItem->dataFileSize;

		} else if (buff == strstr(buff, "Content-Length")) {

			/*
			 * Encountered the Content-Length header on the response.
			 */
			int tmpsize;
			sscanf(buff, "Content-Length: %d", &tmpsize);

			// Save it off in the item structure. Needed for later processing.
			myItem->contentLength = (off_t)tmpsize;

			// Compute custom metadata size
			myItem->cmFileSize = myItem->contentLength - myItem->dataFileSize;

		} else if (buff == strstr(buff, "X-HCP-CustomMetadataFirst")) {

			/*
			 * Encountered the X-HCP-CustomMetadataFirst header on the response.
			 */
			char tmpBoolString[10]; // should only have to hold "true" or "false" value

			sscanf(buff, "X-HCP-CustomMetadataFirst: %s", tmpBoolString);
			if (0 == strcmp(tmpBoolString, "true")) {
				myItem->bCustomMetadataFirst = TRUE;
			} else {
				myItem->bCustomMetadataFirst = FALSE;
			}
		}
	}

	// Dump out all headers for debugging purposes.
	printf("%s\n", buff);

	return nmemb*size;
}

/**
 *  This call back function is called by CURL to request the content to be
 *  written somewhere in the context of an HTTP GET request. Since this is
 *  Whole Object I/O, the stream contains both object data and custom metadata.
 *  This routine will determine which local file the stream bytes need to be
 *  written to.
 *
 *  For a CURL GET, either the object data or the custom metadata can come first.
 *  It depends on if the GET was performed specifying the X-HCP-CustomMetadataFirst
 *  HTTP header with a "true" value.  Because of this feature, this routine is a bit
 *  more complex and focuses on "first" and "second" files.
 */
size_t wholeio_write_cb(void *ptr, size_t size, size_t nmemb, void *userdata) {
	size_t retval = 0;

	item_t *pMyItem = (item_t *)userdata;

	FILE *firstFile, *secondFile;
	size_t firstFileRemaining = 0;

	/*
	 * First, it is necessary to setup what is the first file and second file in
	 *   the stream.  Based on this determination, need to also compute how many
	 *   bytes are remaining to be written in the first file.
	 *
	 *   The bCustomMetadataFirst boolean was set while processing the HTTP headers
	 *   before the stream is processed resulting in this callback to be executed.
	 */
	if ( ! pMyItem->bCustomMetadataFirst) {
		//
		// Data File is first.
		//
		firstFile = pMyItem->localDataFile;
		secondFile = pMyItem->localCMFile;

		// Only compute the content remaining in the first file if we haven't
		//  already finished it.
		if ( ! pMyItem->bFirstFileComplete )
			firstFileRemaining = pMyItem->contentLength - pMyItem->cmFileSize;
	} else {
		//
		// Custom Metadata File is first.
		//
		firstFile = pMyItem->localCMFile;
		secondFile = pMyItem->localDataFile;

		// Only compute the content remaining in the first file if we haven't
		//  already finished it.
		if ( ! pMyItem->bFirstFileComplete )
			firstFileRemaining = pMyItem->contentLength - pMyItem->dataFileSize;
	}

	/*
	 * Process any content to the first file if there is still some remaining in the stream.
	 *   It may be the case that we already processed the file or it is a zero length file.
	 */
	if ( ! pMyItem->bFirstFileComplete && 0 < firstFileRemaining) {

		// Compute the max we can write, it is the smaller of what is passed into this function
		//  and what is remaining for the first file.
		size_t maxToWrite = (size * nmemb) > firstFileRemaining ? firstFileRemaining : (size * nmemb);

		// Write as much as we can.
		retval = fwrite(ptr, 1, maxToWrite, firstFile);

		// Make sure we didn't run into some error.
		if (ferror(firstFile) || retval != maxToWrite) {
			return (-1); // Ouch!!
		}

		// Decrement the contentLength size in the item structure. We are going to
		//  use that to keep track how much still needs to be written between calls
		//  to this call back function.
		pMyItem->contentLength -= retval;

		// See if we reached the end of the first file that we are expecting.
		if (firstFileRemaining == maxToWrite) {
			pMyItem->bFirstFileComplete = TRUE;
		}
	}

	// Don't want an else here.  We might fall through after reading the last of the
	//  first file from the stream.

	/*
	 * Process any content for the second file in the stream.
	 */
	if (pMyItem->bFirstFileComplete) {
		// Get busy and write what is remaining in the current stream buffer.
		retval += fwrite(ptr + retval, 1, (size * nmemb) - retval, secondFile);

		// Make sure we didn't run into some error.
		if (ferror(firstFile)) {
			retval = -1;  // Adjust the return value into a failure.
		}
	}

	return retval;
}

/**
 *  This call back function is called by CURL to request the content to be
 *  read from somewhere in the context of an HTTP PUT request. This routine
 *  will be responsible for figuring out where the requested content should
 *  be read from, either the data file or the custom metadata file based on
 *  where in the stream we are.
 *
 *  For a CURL PUT, the data file ALWAYS comes before the custom metadata
 *  file.
 */
size_t wholeio_read_cb(void *ptr, size_t size, size_t nmemb, void *userdata) {
	size_t retval = 0;

	item_t *pMyItem = (item_t *)userdata;

	/*
	 * Process content to the object data file if there is any more to
	 *  process.
	 */
	if ( ! pMyItem->bFirstFileComplete ) {
		// Yes, try and read as much as we can into this buffer.
		retval = fread(ptr, size, nmemb, pMyItem->localDataFile);

		// If we encountered an error, make sure we end here.
		if (ferror(pMyItem->localDataFile)) {
			return CURL_READFUNC_ABORT;
		}

		// if our last read triggered an feof
		if (feof(pMyItem->localDataFile)) {
			pMyItem->bFirstFileComplete = TRUE;
		}
	}

	/*
	 * Process content to the custom metadata file if we are done pushing
	 * the object data onto the stream.
	 */
	if ( pMyItem->bFirstFileComplete && 0 == retval ) {
		// Yes.  Read from CM file.
		retval = fread(ptr, size, nmemb, pMyItem->localCMFile);

		// Make sure we aren't in trouble..
		if (ferror(pMyItem->localCMFile)) {
			retval = CURL_READFUNC_ABORT;
		}
	}

	return retval;
}

/**
 * This routine utilizes the libcurl library to perform and HTTP PUT of an object and
 * its custom metadata using the Whole I/O mechanism.
 */
int do_write(CURL *curl, const char *srcDataFilePath, const char *srcCMFilePath,
		const char *dstURI, const char *authHeader) {

	int retval = EXIT_SUCCESS;
	struct curl_slist *httpHeaderList = NULL;

	/***
	 *** Setup our "item" structure that will hold the source file information and
	 *** utilized by the libcurl callback functions.
	 ***/
	item_t myItem;
	memset(&myItem, 0, sizeof(myItem));

	// Open data file to be sent.
	myItem.localDataFile = fopen(srcDataFilePath, "rb");
	if(!myItem.localDataFile) {
		fprintf(stderr, "Unable to open data file %s\n", srcDataFilePath);
		_exit(EXIT_FAILURE);
	}

	// Get the size of the data file and put in the item structure.
	struct stat buf;
	if (-1 == stat(srcDataFilePath, &buf)) {
		fprintf(stderr, "Unable to obtain size of data file. errno(%d)\n", errno);
		_exit(EXIT_FAILURE);
	}
	myItem.dataFileSize = buf.st_size;

	// Open Custom Metadata file to be sent.
	myItem.localCMFile = fopen(srcCMFilePath, "rb");
	if(!myItem.localCMFile) {
		fprintf(stderr, "Unable to open custom metadata file %s\n", srcCMFilePath);
		_exit(EXIT_FAILURE);
	}

	// Get the size of the custom metadata file and put in the item structure.
	if (-1 == stat(srcCMFilePath, &buf)) {
		fprintf(stderr, "Unable to obtain size of custom metadata file. errno(%d)\n", errno);
		_exit(EXIT_FAILURE);
	}
	myItem.cmFileSize = (size_t)buf.st_size;

	// Compute the overall content length from the data and CM file sizes.
	myItem.contentLength = myItem.dataFileSize + myItem.cmFileSize;


	/***
	 *** Setup the CURL object for performing the PUT request.
	 ***/

	// Reset curl to clear any prior usage.
	curl_easy_reset(curl);

	// Bypass any SSL Key checks because we are not using site specific keys.
    //   Note: Not needed if only doing HTTP
	curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
	curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 1L);

	// Provide the response header call back function. It will be executed when the
	//  HTTP headers are being processed in the HTTP response.
	curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &myHeaderProcessor);

	// Set object as an HTTP "PUT" operation.
	curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);  // Note: CURLOPT_PUT is depreciated in 7.12.1

	// Set the HCP URL that will be executed.  This includes the "type=whole-object"
	//  parameter.
	curl_easy_setopt(curl, CURLOPT_URL, dstURI);

	// Setup the read callback function and user data that will be used to
	//   obtain the content from the files to include in the message-body of
	//   the HTTP request.
	curl_easy_setopt(curl, CURLOPT_READFUNCTION, wholeio_read_cb);
	curl_easy_setopt(curl, CURLOPT_READDATA, (void *)&myItem);

	/*
	 * Setup the header to indicate how big the data portion of the
	 *  Whole I/O is as well as providing the Authorization header.
	 */

	char HCPSizeHeader[80];
	sprintf(HCPSizeHeader, "X-HCP-Size: %d", (int)myItem.dataFileSize);
	httpHeaderList = curl_slist_append(httpHeaderList, HCPSizeHeader);

	// Use this HCP Authorization header to perform authentication.
	httpHeaderList = curl_slist_append(httpHeaderList, authHeader);

	// Set the headers.
	curl_easy_setopt(curl, CURLOPT_HTTPHEADER, httpHeaderList);

	/***
	 *** Execute the HTTP PUT request.
	 ***/

	CURLcode curlRetVal = curl_easy_perform(curl);
	if(curlRetVal != CURLE_OK) {
		fprintf(stderr, "Error writing %s to HCP!\nReturn Code: %d\n", srcDataFilePath, curlRetVal);
		retval = EXIT_FAILURE;
		goto cleanup;
	}

	/***
	 *** Process the HTTP Response.
	 ***/

	long responseCode = 0;

	curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode);

	// Any code not in the 200 range is a failure.
	if (2 != responseCode / 100) {
		fprintf(stderr, "ERROR: Unexpected response code from server (%d). Operation failed.\n", (int)responseCode);
		retval = EXIT_FAILURE;
	}


  cleanup:

    /*
     * Close any open files.
     */
    if (NULL != myItem.localDataFile) {
      fclose(myItem.localDataFile);
    }

    if (NULL != myItem.localCMFile) {
	  fclose(myItem.localCMFile);
    }

    /*
     *  Even though the curl handle has a pointer to it, we still own this list.
     */
    if (httpHeaderList) {
    	curl_slist_free_all(httpHeaderList);

    	// Clear out the HTTP Header pointer we setup just in case this curl connection
    	// is used for another follow up operation.
    	curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL);
    }

	return retval;
}

/**
 * This routine performs and HTTP GET of an object and its custom metadata using the
 * Whole I/O mechanism.
 *
 * Note that there is some conditionally compiled code to show how to request the
 * custom metadata before the object data in the stream.
 */
int do_read(CURL *curl, const char *srcFileURI, const char *dstDataFilePath, const char *dstCMFilePath,
		    const char *authHeader) {

	int retval = EXIT_SUCCESS;
	struct curl_slist *httpHeaderList = NULL;

	/***
	 *** Setup our "item" structure that will hold the destination file information and
	 *** utilized by the libcurl callback functions.
	 ***/

	item_t myItem;
	memset(&myItem, 0, sizeof(myItem));

	// Open data file to be sent.
	myItem.localDataFile = fopen(dstDataFilePath, "wb");
	if(!myItem.localDataFile) {
		fprintf(stderr, "Unable to open data file %s\n", dstDataFilePath);
		_exit(EXIT_FAILURE);
	}

	// Open Custom Metadata file to be sent.
	myItem.localCMFile = fopen(dstCMFilePath, "wb");
	if(!myItem.localCMFile) {
		fprintf(stderr, "Unable to open custom metadata file %s\n", dstCMFilePath);
		_exit(EXIT_FAILURE);
	}

#ifdef PERFORM_CM_FIRST
	myItem.bCustomMetadataFirst = TRUE;
#endif

	/***
	 *** Setup the CURL object for performing the GET request.
	 ***/

	// Reset curl to clear out prior usage.
	curl_easy_reset(curl);

	//  Bypass any SSL Key checks because we are not using site specific keys.
    //     Not needed if only doing HTTP
	curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
	curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 1L);

	// Set object to do an HTTP GET operation.
	curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L);

	// Provide the response header call back function. It will be executed when the
	//  HTTP headers are being processed in the HTTP response.
	//
	// This part is important.  The item structure will be passed to the HTTP
	//   header processor because the header processor will be computing the
	//   object data and custom metadata sizes, as well as indicating what order
	//   the object data and custom metadata will be in the stream returned by
	//   the Whole I/O HTTP GET request.
	curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, &myHeaderProcessor);
	curl_easy_setopt(curl, CURLOPT_HEADERDATA, &myItem);

	//
	// This callback function will handle the processing of writing both
	// the object data file and the custom metadata file from the one stream
	// passed back from the HTTP GET request.
	curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, wholeio_write_cb);
	curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)(&myItem));

	// Set the HCP URL that will be executed.  This includes the "type=whole-object"
	//  parameter.
	curl_easy_setopt(curl, CURLOPT_URL, srcFileURI);

	// Use this HCP Authorization header to perform authentication.
	httpHeaderList = curl_slist_append(httpHeaderList, authHeader);

#ifdef PERFORM_CM_FIRST

	//
	// Setup the header to indicate we want the custom metadata first in the stream.
	//  As a default the object data is sent back first, this will tell HCP to send
	//  the custom metadata first.  The benefit of this would be to allow the applica-
	//  tion to peek at its custom metadata that may help dictate what needs to be
	//  done with the object data that will follow.
	struct curl_slist *httpHeaderList = NULL;
	httpHeaderList = curl_slist_append(httpHeaderList, "X-HCP-CustomMetadataFirst: true");

#endif

	curl_easy_setopt(curl, CURLOPT_HTTPHEADER, httpHeaderList);

	/***
	 *** Execute the HTTP GET request.
	 ***/

	CURLcode curlRetVal = curl_easy_perform(curl);
	if(curlRetVal != CURLE_OK) {
		fprintf(stderr, "Error reading %s from HCP!\nReturn Code: %d\n", srcFileURI, curlRetVal);
		retval = EXIT_FAILURE;
		goto cleanup;
	}

	/***
	 *** Process the HTTP Response.
	 ***/
	long responseCode = 0;

	curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &responseCode);

	// Make sure we get the appropriate 200 level HTTP status code.
	if (2 != responseCode / 100) {
		fprintf(stderr, "ERROR: Unexpected response code from server (%d). Operation failed.\n", (int)responseCode);
		retval = EXIT_FAILURE;
	}

  cleanup:

    /*
     * Close any open files.
     */
    if (NULL != myItem.localDataFile) {
      fclose(myItem.localDataFile);

      if (EXIT_SUCCESS != retval)
        unlink(dstDataFilePath);  // Remove any residual file.
    }

    if (NULL != myItem.localCMFile) {
  	  fclose(myItem.localCMFile);

      if (EXIT_SUCCESS != retval)
        unlink(dstCMFilePath);  // Remove any residual file.
    }

    /*
     *  Even though the curl handle has a pointer to it, we still own this list.
     */
    if (httpHeaderList) {
    	curl_slist_free_all(httpHeaderList);

    	// Clear out the HTTP Header pointer we setup just in case this curl connection
    	// is used for another follow up operation.
    	curl_easy_setopt(curl, CURLOPT_HTTPHEADER, NULL);
    }

	return retval;
}

/**
 * This method is the main line for an example on reading (GET) and writing
 * (PUT) Whole I/O content with HCP.
 *
 * This is only supported in HCP 4.1 and newer.
 */
int main(int argc, char **argv) {
	int retval = EXIT_SUCCESS;

	/****
	 **** Make sure we have the required parameters.
	 ****/
	if (argc != 4) {
		char *basename = strrchr(argv[0], '/');
		if (NULL == basename)
			basename = argv[0];
		else
			basename++;

		fprintf(stderr, "\nUsage: %s <DNS-Namespace> <UserName> <Password>\n", basename);
		fprintf(stderr, "  where <DNS-Namespace> is the fully qualified DNS name of the HCP Namespace.\n");
		fprintf(stderr, "              For example: \"ns1.ten1.myhcp.example.com\"\n");
		fprintf(stderr, "        <Username> and <Password> are the credentials of the HCP Data Access User\n");
		fprintf(stderr, "              for the namespace.\n\n");

		exit(EXIT_FAILURE);
	}

	/****
	 **** Initialize the execution environment for HTTP over libcurl
	 ****/

	/* platform-specific setup */
	curl_global_init(CURL_GLOBAL_ALL);

	/* Build the HCP file path that include the URI parameter specifying Whole I/O. */
	char *HCPFileURI = malloc( strlen(HCP_PROTOCOL) + 3
			+ strlen(argv[1]) + 1 + strlen(HCP_FILE_PATH)
			+ strlen(INPUT_DATA_FILE_NAME) + 1
			+ strlen(HCP_WHOLE_IO_PARAM) + 1 );
	sprintf(HCPFileURI, "%s://%s/%s/%s?%s", HCP_PROTOCOL, argv[1],
			HCP_FILE_PATH, INPUT_DATA_FILE_NAME, HCP_WHOLE_IO_PARAM);

	CURL *curl = NULL; /* curl session handle */

	// Use this HCP security cookie to perform authentication.
	char *authHeader = createAuthHeader(argv[2], argv[3]);

	/*
	 * Create a curl session to use for the operations.
	 */
	curl = curl_easy_init();
	if(!curl) {
		fprintf(stderr, "Unable to initialize cURL HTTP library\n");

		retval = EXIT_FAILURE;
		goto cleanup;
	}

	/****
	 **** Perform a write (PUT) of data and custom metadata content on HCP.
	 ****/
	retval = do_write(curl, INPUT_DATA_FILE_NAME, INPUT_CM_FILE_NAME, HCPFileURI,
			          authHeader);
	if (EXIT_SUCCESS != retval) {
		fprintf(stderr, "FAILURE: do_write() failed!!\n");

		retval = EXIT_FAILURE;
		goto cleanup;
	}

	/****
	 **** Perform a read (GET) of data and custom metadata content from HCP.
	 ****/
	retval = do_read(curl, HCPFileURI, OUTPUT_FILE_NAME, OUTPUT_CM_FILE_NAME,
			         authHeader);
	if (EXIT_SUCCESS != retval) {
		fprintf(stderr, "FAILURE: do_read() failed!!\n");

		retval = EXIT_FAILURE;
		goto cleanup;
	}

	/****
	 **** SUCCESS!!  Just clean-up and get outta here.
	 ****/
cleanup:
	if (HCPFileURI) free(HCPFileURI);

	if (authHeader) free(authHeader);

    /* free curl session */
	if(curl) {
		curl_easy_cleanup(curl);
		curl = NULL;
	}

	/* free platform bindings */
	curl_global_cleanup();

	return EXIT_SUCCESS;
}
